# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import random
import requests
from scrapy import signals
from fake_useragent import UserAgent
ua = UserAgent()

# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter


# 爬虫中间件
class ScrapyTestSpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()

        # 绑定信号 ， 爬虫开始  爬虫结束
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)

        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        # print(spider.name, "爬虫输入  --------------", spider.start_urls)
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.

        for i in result:
            # print(i, "爬虫输出  ==============")
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        # print(exception, "爬虫异常.............", response)
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            # print(r, "start_requests  ==============")
            yield r

    def spider_opened(self, spider):
        spider.logger.info("ScrapyTestSpiderMiddleware  Spider opened: %s" % spider.name)

    def spider_closed(self, spider):
        spider.logger.info("ScrapyTestSpiderMiddleware  Spider closed: %s" % spider.name)



# 下载中间件
class ScrapyTestDownloaderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        print(request.url, "请求经过下载中间件")
        # Called for each request that goes through the downloader
        # middleware.
        # request.headers["referer"] = "https://www.baidu.com"
        # request.headers['User-Agent'] = ua.random
        # request.headers['cookies'] = "name=qiku;age=10"
        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.
        print(f"{response}", "响应经过下载中间件")
        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        response.json().update({"state": "醒醒啦"})
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


class RandomUserAgentDownloaderMiddleware:
    def process_request(self, request, spider):
        # 中间件获取settings信息
        self.useragent_list = spider.crawler.settings.get("USERAGENTS")
        request.headers["User-Agent"] = random.choice(self.useragent_list)
        # 使用第三方UA模块
        # request.headers["User-Agent"] = ua.random
        return None


class RandomCookiesDownloaderMiddleware:
    def process_request(self, request, spider):
        # 中间件获取settings信息
        self.usercookies_list = spider.crawler.settings.get("USER_COOKIES")
        request.headers["cookies"] = random.choice(self.usercookies_list)
        return None


class RandomIpDownloaderMiddleware:
    def process_request(self, request, spider):
        # 中间件获取settings信息
        self.ip_list = spider.crawler.settings.get("IP_LIST")
        request.meta['proxy'] = f"http://{random.choice(self.ip_list)}"
        return None


class GETIpDownloaderMiddleware:
    def process_request(self, request, spider):
        # 中间件获取settings信息
        res = requests.get("https://dps.kdlapi.com/api/getdps/?secret_id=?&signature=?&num=1&pt=1&format=text&sep=1&f_auth=1&generateType=4")
        request.meta['proxy'] = f"http://{res.text}"
        return None


class TunnelIpDownloaderMiddleware:
    def process_request(self, request, spider):
        # 中间件获取settings信息
        request.meta['proxy'] = f"http://t14409688113668:hyn69tje@j547.kdltps.com:15818"
        # Scrapy 在使用隧道代理可能会出现复用之前建立的连接导致不能正常更换IP，请在 header 中加入 Connection: close
        request.headers["Connection"] = "close"
        return None
